R Notebook

# load in required libraries
pacman::p_load(
  here,        
  janitor,    
  lubridate,  
  tidyverse,
  skimr
)

# load in raw datasets
airlines <- read_csv(here("raw_data/airlines.csv"))

Rows: 12 Columns: 2
── Column specification ───────────────────────────────────────────────────────────────────────────────────
Delimiter: ","
chr (2): carrier, name

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

airports <- read_csv(here("raw_data/airports.csv"))

Rows: 1333 Columns: 8
── Column specification ───────────────────────────────────────────────────────────────────────────────────
Delimiter: ","
chr (4): faa, name, dst, tzone
dbl (4): lat, lon, alt, tz

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

flights <- read_csv(here("raw_data/flights.csv"))

Rows: 303748 Columns: 19
── Column specification ───────────────────────────────────────────────────────────────────────────────────
Delimiter: ","
chr   (4): carrier, tailnum, origin, dest
dbl  (14): year, month, day, dep_time, sched_dep_time, dep_delay, arr_time, sched_arr_time, arr_delay, ...
dttm  (1): time_hour

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

planes <- read_csv(here("raw_data/planes.csv"))

Rows: 3521 Columns: 9
── Column specification ───────────────────────────────────────────────────────────────────────────────────
Delimiter: ","
chr (5): tailnum, type, manufacturer, model, engine
dbl (4): year, engines, seats, speed

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

weather <- read_csv(here("raw_data/weather.csv"))

Rows: 26201 Columns: 15
── Column specification ───────────────────────────────────────────────────────────────────────────────────
Delimiter: ","
chr   (1): origin
dbl  (13): year, month, day, hour, temp, dewp, humid, wind_dir, wind_speed, wind_gust, precip, pressure...
dttm  (1): time_hour

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

Initial Data Exploration

Use summary to perform an initial exploration of the datasets, noting where there are large numbers of NA values. Variable types will be addressed once we have a cleaned, combined dataset.

# explore the data
summary(airlines)
summary(airports)
summary(planes)
summary(weather)
summary(flights)

skim(flights)
skim(weather)

Initial Data Cleaning

There are a large number of missing values in the weather and flights dataset. The proportion of the dataset with missing values is calcualted below.

# calculate proportion of the data which are NAs
weather %>% 
  summarise_each(funs(100 * mean(is.na(.))))

Warning: `summarise_each_()` was deprecated in dplyr 0.7.0.
Please use `across()` instead.
This warning is displayed once every 8 hours.
Call `lifecycle::last_warnings()` to see where this warning was generated.
Warning: `funs()` was deprecated in dplyr 0.8.0.
Please use a list of either functions or lambdas: 

  # Simple named list: 
  list(mean = mean, median = median)

  # Auto named with `tibble::lst()`: 
  tibble::lst(mean, median)

  # Using lambdas
  list(~ mean(., trim = .2), ~ median(., na.rm = TRUE))
This warning is displayed once every 8 hours.
Call `lifecycle::last_warnings()` to see where this warning was generated.

flights %>% 
  summarise_each(funs(100 * mean(is.na(.))))

The proportion of NAs in the flights dataset is less than 3% and therefore of little concern, however as the NAs are in columns that I will be using in the analysis, I can drop these from the dataset.

# drop NAs from flight data since these observations won't be useful in the analysis
flights <- flights %>% 
  drop_na()

There are variables in the weather dataset which have a proportion of missing values greater than 90%, there is little value in keeping these variables in the data and therefore they can be dropped.

# drop variables where proportion of NAs is greater than 90%
weather <- weather %>% 
  select(-c(temp, dewp, humid, precip, pressure))

I have chosen to drop three columns from the airports dataset - I don’t foresee these variables being of much consequence to the analysis.

airports <- airports %>% 
  select(-c(tz, dst, tzone))

Joining the Datasets

The flights dataset can be joined to the airlines, planes and airports datasets (the join is repeated for the airport dataset so we have corresponding information for both the origin and destination airports)

# join flights dataset with airlines, planes and airport datasets
flights_joined <- flights %>% 
  left_join(airlines, by = "carrier") %>% 
  left_join(planes, by = "tailnum") %>% 
  left_join(airports, by = c("origin" = "faa")) %>% 
  left_join(airports, by = c("dest" = "faa"))
flights_joined

ewr_flights <- flights_joined %>% 
  filter(origin == "EWR")
jfk_flights <- flights_joined %>% 
  filter(origin == "JFK")
lga_flights <- flights_joined %>% 
  filter(origin == "LGA")

ewr_weather <- weather %>% 
  filter(origin == "EWR")
jfk_weather <- weather %>% 
  filter(origin == "JFK")
lga_weather <- weather %>% 
  filter(origin == "LGA")

ewr_flights_joined <- ewr_flights %>% 
  left_join(ewr_weather, by = "time_hour")
ewr_flights_joined

jfk_flights_joined <- jfk_flights %>% 
  left_join(jfk_weather, by = "time_hour")
jfk_flights_joined

lga_flights_joined <- lga_flights %>% 
  left_join(lga_weather, by = "time_hour")
lga_flights_joined

class(x)

[1] "times"

colnames

 [1] "orig_depart_date"  "orig_depart_time"  "sched_depart_time" "dep_delay"         "orig_arr_time"    
 [6] "sched_arr_time"    "arr_delay"         "carrier"           "flight_number"     "tail_number"      
[11] "origin.x"          "origin_name"       "dest"              "flight_time"       "distance"         
[16] "time_hour"         "name.x"            "year"              "type"              "manufacturer"     
[21] "model"             "seats"             "engine"            "name.y"            "lat"              
[26] "lon"               "alt"               "origin.y"          "wind_dir"          "wind_speed"       
[31] "wind_gust"         "visib"